# C:/Python27/TextResuming/model/text.py
# This is model file
# Author : @ismailsunni

'''A modul for text model, including sentence, paragraph, and text itself.'''

from stemming.porter2 import stem
import wrapper as wrapper
from pos_tagger import PosTagger
import Phrase.phrase_chunker as phraseChunker # Used Adit's Phrase Chunker
import head_chunker as hc
import preprocess as pp
import is_parser as is_parser
import word_weighting as ww
import util as util
import os
import sys
import inspect
import nltk
import string
	
class title:
	'''A class for defining a title.'''
	
	def __init__(self, text):
		self.text = text

	def getWords(self):
		'''Return list of word.'''
		
		retval = pp.stem_sentence(self.text)
		retval = pp.normalizeString(retval)
		retval = pp.foldCase(retval)
		retval = pp.removePunctuation(retval)
		#retval = nltk.word_tokenize(retval) #using nltk
		retval = retval.split() #using split function
		retval = pp.removeStopWord(retval) #remove stopword
		return retval
		
	def getNumWords(self):
		'''Return num of word.'''
		
		return len(self.getWords())

	def getPhrase(self):
		'''Return list of phrase.'''
		
		return phraseChunker.extractNounPhrase(phraseChunker.chunkPhrase(phraseChunker.tagSentence(self.text)))

	def get_list_NP_terminal(self):
		'''Return list of NP terminal.
			NP terminal is the last NP in a phrase.'''
		
		parse_tree = phraseChunker.chunkPhrase(phraseChunker.tagSentence(self.text))
		return hc.get_list_NP_terminal(parse_tree)

	def getNumPhrase(self):
		'''Return num of phrase.'''
		
		return len(self.getNumPhrase())

class sentence:
	'''A class for defining a sentence.'''
	
	def __init__(self, idx, text):
		self.idx = idx
		self.text = text

	def getText(self):
		'''Return text of the sentence.'''
		
		return self.text

	def printSentence(self):
		'''Print sentence to the screen.'''
		
		print 'id sentence:' ,self.idx, '\ncontent : ', self.text

	def writeSentence(self, fstream):
		'''Write the sentence to a file.'''
		
		lines = 'id sentence:' + str(self.idx) + '\ncontent:' + self.text + '\n'
		fstream.writelines(lines)

	def getWords(self):
		'''Return list of word.'''    
		
		retval = pp.stem_sentence(self.text)
		retval = pp.foldCase(retval)
		retval = pp.removePunctuation(retval)
		#retval = nltk.word_tokenize(retval) #using nltk
		retval = retval.split() #using split function
		retval = pp.removeStopWord(retval) #remove stopword
		return retval

	def get_head_NP(self):
		'''Return list of head of NP.
			head NP is NP that lead the NP.'''
		
		return hc.get_head_NP(pp.foldCase(pp.removePunctuation(self.text)))

	def get_NP(self):
		'''Return list of NP from sentence.'''
		
		# return hc.get_list_NP_terminal_from_sentence(pp.foldCase(pp.removePunctuation(self.text)))
		# return hc.get_list_NP_terminal_from_sentence(pp.foldCase(self.text))
		return hc.get_list_NP_terminal_from_sentence(self.text)

	def getNumWords(self):
		'''Return num of word.'''
		
		return len(self.getWords())

	def getPhrase(self):
		'''Return list of phrase.'''
		
		return phraseChunker.extractNounPhrase(phraseChunker.chunkPhrase(phraseChunker.tagSentence(self.text)))

	def getNumPhrase(self):
		'''Return num of phrase.'''
		
		return len(self.getNumPhrase())
	
	def parse_INA(self):
		'''Return a list of word and its tag.'''
		
		return wrapper.PosWrapper.parse(self.text)
		
class segment:
	'''A class defining a segment.
		A segment contains list of sentences.'''

	def __init__(self, segment_index, content):
		self.segment_index = segment_index
		self.content = content

	def get_sentence(self, sentence_index):
		'''Return a sentence by sentence_index.'''

		return self.content[sentence_index]

	def get_NP(self):
		'''Return set of NP from a segment.'''
		
		set_NP = set()
		for sent in self.content:
			set_NP |= set(sent.get_NP())

		return set_NP

	def get_list_NP(self):
		'''Return list of NP from a segment.'''
		
		list_NP = []
		for sent in self.content:
			list_NP.extend(sent.get_NP())

		return list_NP

	def get_list_NP_term(self):
		'''Return a list of term in NP. Stopword is removed, and stemmed.'''

		list_NP = self.get_list_NP()
		list_of_term = []
		
		# Create list of term
		for NP in list_NP:
			new_terms = is_parser.sentence_to_list_of_word(NP)
			list_of_term.extend(new_terms)

		# Remove stopwords
		list_NP_wout_stopword = pp.removeStopWord(list_of_term)

		# Stemming
		retval = pp.stem_list_of_word(list_NP_wout_stopword)

		return retval

	def is_subtitle(self):
		'''True if it is subtitle.
			Indicate by one sentence without full stop.'''
		
		return False
		
		if self.get_num_sentences() == 1:
			last_sentence = self.get_sentence(-1)
			last_char = last_sentence.getText()[-2]	# last char is new line
			new_line = last_sentence.getText()[-1]	# last char is new line

			if last_char == '.' or last_char == '?' or last_char == '!' or last_char == '"' or last_char == "'" or last_char == " ":
				return False

			elif new_line != '\n':
				#	if new_line != \n it is a last segment
				return False

			else:
				return True

		else:
			return False

	def get_num_sentences(self):
		'''Return the number of sentences.'''

		return len(self.content)

	def create_base_list_of_term_value_segment(self, dict_NP_term):
		'''Return a base list of term value.'''

		list_term = self.get_list_NP_term()
		retval = ww.get_dict_fraction(list_term, dict_NP_term)
		retval = util.delete_dict_by_value(retval, 1)	# remove value = 1
		# retval = util.sort_dictionary_by_value(retval)	# sort

		return retval

	def print_segment(self):
		'''Print function.'''
		
		print 'Segment index : ', self.segment_index
		for sent in self.content:
			sent.printSentence()

	def get_content(self):
		'''Return the content.'''
		
		content = ''
		for sent in self.content:
			content += sent.getText()
			content += ' '

		return content

	# Indonesian Part
	def get_word_INA_tagged(self, list_tag = []):
		'''Return list of word whose tag in list_tag. If list_tag = None, return all word.'''
		
		list_word_tagged = []
		for sent in self.content:
			list_word_tagged.extend(sent.parse_INA())
		
		retval = []
		if list_tag == [] or list_tag == None:
			for word_tagged in list_word_tagged:
				retval.append(word_tagged[0])
		else:
			for word_tagged in list_word_tagged:
				if word_tagged[1] in list_tag:
					retval.append(word_tagged[0])
		
		return retval
		
class segmented_news:
	'''A class defining a news with segmentations.'''
	
	def __init__(self, title, source, content):
		self.title = title
		self.source = source
		self.content = content

	def print_segmented_news(self):
		'''Print function.'''
		
		print 'Title : ', self.title
		print 'Source : ', self.source
		i = 0
		for seg in self.content:
			seg.print_segment()

	def get_base_list_term(self):
		'''Return list of list term in each segment.'''

		retval = []
		for segm in self.content:
			retval.append(segm.get_list_NP_term())

		return retval

	def get_list_NP_term(self):
		'''Return list of term in segmented news.'''

		retval = []
		for segm in self.content:
			retval.extend(segm.get_list_NP_term())

		return retval

	def get_base_set_similarity_term(self):
		'''Create list of segment contains terms.'''

		list_of_segment = []
		for segm in self.content:
			list_segment = []
			list_segment.append([segm.segment_index])
			list_segment.append(set(segm.get_list_NP_term()))
			list_of_segment.append(list_segment)

		return list_of_segment

	def create_base_set_similarity(self):
		'''Create list of segment for base in set similarity.'''
		
		list_of_segment = []
		for segm in self.content:
			list_segment = []
			list_segment.append([segm.segment_index])
			list_segment.append(segm.get_NP())
			list_of_segment.append(list_segment)

		return list_of_segment

	def create_base_list_of_term_value(self, dict_NP_term):
		'''Return a base list of term value.
			
			@parameters:
			@dict_NP_term : dictionary of NP term and its value.'''

		retval = []
		for segm in self.content:
			retval.append(segm.create_base_list_of_term_value_segment(dict_NP_term))

		return retval

	def get_list_of_NP(self):
		'''Return a list of all NP that is used in the segmented_news.'''
		
		retval = []
		for segm in self.content:
			retval.extend(segm.get_list_NP())
		
		return retval

	def get_set_of_NP(self):
		'''Return a set of all NP that is used in the segmented_news.'''
		
		retval = set()
		for segm in self.content:
			retval |= set(segm.get_NP())
		
		return retval
		
	def get_list_of_list_NP(self):
		'''Return a list of list of all NP that is used in each segment.'''
		
		retval = []
		i = 0
		for segm in self.content:
			retval.append(segm.get_list_NP())
		
		return retval

	def create_dict_term_TF_IDF(self):
		'''Return a dictionary contain term and its TfIdf value.'''
		
		list_term = self.get_list_NP_term()
		set_term = set(list_term)
		list_of_list_of_term = self.get_base_list_term()

		dict_term_TF_IDF = {}
		for term in set_term:
			TfIdf = ww.get_TF_IDF(term, list_term, list_of_list_of_term)
			dict_term_TF_IDF[term] = TfIdf

		return dict_term_TF_IDF
	
	def create_dict_TF_IDF(self):
		'''Return a dictionary contain phrase and its TfIdf value.'''

		list_NP = self.get_list_of_NP()
		set_NP = self.get_set_of_NP()
		list_of_list_of_NP = self.get_list_of_list_NP()

		dict_TF_IDF = dict()

		for NP in set_NP:
			TfIdf = ww.get_TF_IDF(NP, list_NP, list_of_list_of_NP)
			dict_TF_IDF[NP] = TfIdf

		return dict_TF_IDF

	def create_dict_NP_term(self):
		'''Return a dictionary contains term and number of it.'''

		# Create list_of_NP_term
		list_of_NP_term = []
		for segm in self.content:
			list_of_NP_term.extend(segm.get_list_NP_term())
		
		# Create the dictionary
		dict_NP_term = {}
		for term in set(list_of_NP_term):
			dict_NP_term[term] = list_of_NP_term.count(term)

		return dict_NP_term

	def get_num_segment(self):
		'''Return the number of segment.'''

		return len(self.content)

	def get_segment(self, idx):
		'''Return the segment.'''

		return self.content[idx]

	def save_segmented(self, file_path, list_cluster = None):
		'''Save segmented_news to a file with formated, if list_cluster == None, only with index.'''
		
		if list_cluster == None:
			list_to_be_wrote = []
			list_to_be_wrote.append(self.title)
			# list_to_be_wrote.append('\n')
			list_to_be_wrote.append(self.source)
			list_to_be_wrote.append('\n')
			
			i = 0
			for segm in self.content:
				if segm.get_content()[-2] == '\n':
					list_to_be_wrote.append(('[' + str(i) + ']' + segm.get_content())[:-1])
				else:
					list_to_be_wrote.append(('[' + str(i) + ']' + segm.get_content()))
				i += 1

			return util.write_list(file_path, list_to_be_wrote, True)

		else:
			list_to_be_wrote = []
			list_to_be_wrote.append(self.title)
			list_to_be_wrote.append('\n')
			list_to_be_wrote.append(self.source)
			list_to_be_wrote.append('\n')

			for cluster in list_cluster:
				list_to_be_wrote.append('<p>\n')
				for idx_segm in cluster:
					if idx_segm == cluster[0]:	# main sentence
						if self.get_segment(idx_segm).get_content()[-2] == '\n':
							list_to_be_wrote.append(('<ms>[' + str(idx_segm) + ']' + self.get_segment(idx_segm).get_content())[:-2] + '</ms>\n')
						else:
							list_to_be_wrote.append('<ms>[' + str(idx_segm) + ']' + self.get_segment(idx_segm).get_content() + '<ms>\n')	

					else:
						if self.get_segment(idx_segm).get_content()[-2] == '\n':
							list_to_be_wrote.append(('[' + str(idx_segm) + ']' + self.get_segment(idx_segm).get_content())[:-1])
						else:
							list_to_be_wrote.append('[' + str(idx_segm) + ']' + self.get_segment(idx_segm).get_content() + '\n')
							
				list_to_be_wrote.append('</p>\n')

			return util.write_list(file_path, list_to_be_wrote, False)
			pass

	# Indonesian Part
	def get_list_of_list_word_INA_tagged(self, list_tag = []):
		'''Return list of list word in segment whose tag in list_tag.'''
		
		retval = []
		for segm in self.content:
			retval.append(segm.get_word_INA_tagged(list_tag))
		self.list_tag = list_tag
		self.list_of_list_word_INA_tagged = retval

		return retval
		
	def get_dict_TF_IDF_word_INA_tagged(self, list_tag = []):
		'''create dictionary TF IDF for word INA tagged.'''
		
		if self.list_of_list_word_INA_tagged == None:
			self.get_list_of_list_word_INA_tagged(list_tag)
		
		list_of_word_INA_tagged = []
		for segm_word_INA_tagged in self.list_of_list_word_INA_tagged:
			list_of_word_INA_tagged.extend(segm_word_INA_tagged)
		
		set_of_word_INA_tagged = set(list_of_word_INA_tagged)
		
		dict_TF_IDF_word_INA_tagged = {}
		for word in set_of_word_INA_tagged:
			TfIdf = ww.get_TF_IDF(word, list_of_word_INA_tagged, self.list_of_list_word_INA_tagged)
			dict_TF_IDF_word_INA_tagged[word] = TfIdf
		
		return dict_TF_IDF_word_INA_tagged	
			
def create_segment(segment_index, real_segment):
	'''Creating segment object from list of string.'''

	list_sentence = is_parser.parse_text_to_sentences(real_segment)
	content = []
	sentence_index = 0
	for s in list_sentence:
		if (len(s) > 2):
			new_sentence = sentence(sentence_index, s)
			content.append(new_sentence)
			sentence_index += 1

	return segment(segment_index, content)

def create_segmented_news(list_of_string):
	'''Create news from list_of_string.'''

	news_title = list_of_string[0]
	news_source = list_of_string[1]

	segment_index = 0
	list_segment = []
	for real_segment in list_of_string[2:]:
		new_segment = create_segment(segment_index, real_segment)
		if not new_segment.is_subtitle():
			list_segment.append(new_segment)
			segment_index += 1

	return segmented_news(news_title, news_source, list_segment)


# test function	
def main3():
	util.debug('main3')
	file_path = 'D:/Proyek/Text Resuming/TextResuming/test/Content Berita 2/_News1.txt'
	# file_path = 'D:/Proyek/Text Resuming/TextResuming/test/News_Ing/2News.txt'
	list_of_string = is_parser.load_segmented_news(file_path)

	segmented_news1 = create_segmented_news(list_of_string)

	list_cluster = [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [21, 22, 23, 24, 25, 26, 27]]
	print segmented_news1.save_segmented('ab.txt', list_cluster)

def main4():
	a = 'But Nabil el-Arabi, the league\'s chairman, said he had received a message from Russian Foreign Minister Sergei Lavrov that expressed support for the league\'s efforts and an expanded "observer" mission.'
	sent_a = sentence(0,a)
	list_NP = sent_a.get_NP()
	print a
	print 'NP'
	util.print_index_list_dict(list_NP)

	list_sent = util.read_text_file('lala.txt')
	sent_b = sentence(0,list_sent[0])
	sent_b.printSentence()
	print list_sent[0]
	list_NPx = sent_b.get_NP()
	print 'NP'
	util.print_index_list_dict(list_NPx)
	
def main_INA_parse():
	a = "Pengprov PBSI Bali gagal merealisasikan pembangunan gedung olahraga (GOR) bulu tangkis bertaraf internasional di masa kepemimpinan Wayan Candra karena tidak bisa mencari lahan."
	b = "Kini PR kembali menimpa Pengprov PBSI Bali yang dikendalikan anggota DPD RI, Nengah Wiratha."
	c = "Saat kepengurusan Wayan Candra yang juga bupati Klungkung, Bali ditantang mempersiapkan lahan, sedangkan biaya pembangunan ditanggung Pengurus Besar (PB) PBSI."
	d = "Namun, hingga kepengurusan itu berakhir, GOR tidak bisa terwujud."
	e = "Respons pembangunan GOR bulu tangkis bertaraf internasional sangat direspons oleh anggota DPR RI asal Bali, Wayan Koster, yang bertindak sebagai pembina Pengprov PBSI Bali."
	f = "Pernyataan sokongan dana diperjuangkan Koster ketika menerina pengurus PBSI Bali di Jakarta, 9 Januari lalu."

	sent_a = sentence(0, a)
	sent_b = sentence(0, b)
	sent_c = sentence(0, c)
	sent_d = sentence(0, d)
	sent_e = sentence(0, e)
	sent_f = sentence(1, f)
	
	segm_1 = segment(0, [sent_a])
	segm_2 = segment(1, [sent_b])
	segm_3 = segment(2, [sent_c])
	segm_4 = segment(3, [sent_d])
	segm_5 = segment(4, [sent_e, sent_f])
	
	snews = segmented_news('', '', [segm_1, segm_2, segm_3, segm_4, segm_5])
	snews.print_segmented_news()
	
	list_tag = ['NN', 'NNP']
	print snews.get_list_of_list_word_INA_tagged(list_tag)
	print snews.get_dict_TF_IDF_word_INA_tagged(list_tag)
	
if __name__ == '__main__':
	#main_INA_parse()
	#main4()
	main3()